{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "id": "uqRzgo0N5PKL"
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import tensorflow as tf\n",
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "import tensorflow.contrib.slim as slim\n",
    "\n",
    "mnist = input_data.read_data_sets('/tmp/mnist/data',one_hot=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 322
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 3146,
     "status": "ok",
     "timestamp": 1532268918360,
     "user": {
      "displayName": "heihei ಠ౪ಠ",
      "photoUrl": "//lh5.googleusercontent.com/-t8laan2e9zs/AAAAAAAAAAI/AAAAAAAAAEY/ub2lsDE03Xk/s50-c-k-no/photo.jpg",
      "userId": "117235303177047266583"
     },
     "user_tz": -480
    },
    "id": "S8OtAmGP476G",
    "outputId": "e1abb213-57b1-4fec-8753-6e5caea66b06"
   },
   "outputs": [],
   "source": [
    "sess = tf.InteractiveSession()\n",
    "\n",
    "x = tf.placeholder(tf.float32, shape=[None, 28*28], name='input')\n",
    "y = tf.placeholder(tf.float32, shape=[None, 10], name='label')\n",
    "is_training = tf.placeholder(tf.bool, name='is_training')\n",
    "learning_rate = tf.placeholder(tf.float32)\n",
    "##  Input : 28x28x1 \n",
    "##  Conv => Pooling : 28x28x1 => 28x28x20 => 14x14x20\n",
    "##  Conv => Pooling : 14x14x20 => 10x10x40 => 5x5x40\n",
    "##  Reshape: 5x5x40 => 1000x1\n",
    "##  FC : 1000x1\n",
    "##  Dropout\n",
    "##  FC : 1000x1\n",
    "##  Dropout\n",
    "##  FC : 1000 => 10\n",
    "def convnet(inputs, is_training, scope):\n",
    "    with tf.variable_scope(scope):\n",
    "        inputs= tf.reshape(inputs, [-1,28,28,1])\n",
    "        \n",
    "        net = slim.conv2d(inputs, 20, [5, 5], padding='SAME', scope='layer1-conv')\n",
    "        net = slim.max_pool2d(net, 2, stride=2, scope='layer2-max-pool')\n",
    "       \n",
    "        net = slim.conv2d(net, 40, [5, 5], padding='VALID', scope='layer3-conv')\n",
    "        net = slim.max_pool2d(net, 2, stride=2, scope='layer4-max-pool')\n",
    "       \n",
    "        net = slim.flatten(net)\n",
    "        \n",
    "        net = slim.fully_connected(net, 1000, scope='layer5',activation_fn=tf.nn.crelu)\n",
    "        net = slim.dropout(net,keep_prob=0.5, is_training=is_training, scope='layer5-dropout')\n",
    "\n",
    "        net = slim.fully_connected(net, 1000, scope='layer6',activation_fn=tf.nn.crelu)\n",
    "        net = slim.dropout(net,keep_prob=0.5, is_training=is_training, scope='layer6-dropout')\n",
    "\n",
    "        net = slim.fully_connected(net, 10, scope='output',activation_fn=tf.nn.softmax)\n",
    "\n",
    "        return net\n",
    "\n",
    "logits = convnet(x, is_training, scope='Covnet')\n",
    "\n",
    "with tf.name_scope('Loss'):\n",
    "    cross_entropy =  slim.losses.softmax_cross_entropy(logits, y)\n",
    "with tf.name_scope('Prediction'):\n",
    "    correct_prediction = tf.equal(tf.argmax(logits, 1), tf.argmax(y, 1))\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "with tf.name_scope('Optimizer'):\n",
    "    train_step = tf.train.AdadeltaOptimizer(learning_rate).minimize(cross_entropy)\n",
    "\n",
    "sess.run(tf.global_variables_initializer())\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 538
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 357654,
     "status": "ok",
     "timestamp": 1532269276082,
     "user": {
      "displayName": "heihei ಠ౪ಠ",
      "photoUrl": "//lh5.googleusercontent.com/-t8laan2e9zs/AAAAAAAAAAI/AAAAAAAAAEY/ub2lsDE03Xk/s50-c-k-no/photo.jpg",
      "userId": "117235303177047266583"
     },
     "user_tz": -480
    },
    "id": "BZFmAY2t-A2Z",
    "outputId": "e24f3b43-7523-4b79-df0d-47512d01f92f"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step:     0,CV Accuracy = 10.42%\n",
      "Step:  1000,CV Accuracy = 97.68%\n",
      "Step:  2000,CV Accuracy = 98.32%\n",
      "Step:  3000,CV Accuracy = 98.48%\n",
      "Step:  4000,CV Accuracy = 98.80%\n",
      "Step:  5000,CV Accuracy = 98.82%\n",
      "Step:  6000,CV Accuracy = 98.82%\n",
      "Step:  7000,CV Accuracy = 99.10%\n",
      "Step:  8000,CV Accuracy = 99.18%\n",
      "Step:  9000,CV Accuracy = 99.06%\n",
      "Step: 10000,CV Accuracy = 98.96%\n",
      "Step: 11000,CV Accuracy = 99.16%\n",
      "Step: 12000,CV Accuracy = 99.20%\n",
      "Step: 13000,CV Accuracy = 99.18%\n",
      "Step: 14000,CV Accuracy = 99.20%\n",
      "Step: 15000,CV Accuracy = 99.16%\n",
      "Step: 16000,CV Accuracy = 99.18%\n",
      "Step: 17000,CV Accuracy = 99.18%\n",
      "Step: 18000,CV Accuracy = 99.30%\n",
      "Step: 19000,CV Accuracy = 99.18%\n",
      "Step: 20000,CV Accuracy = 99.20%\n",
      "Step: 21000,CV Accuracy = 99.34%\n",
      "Step: 22000,CV Accuracy = 99.30%\n",
      "Step: 23000,CV Accuracy = 99.20%\n",
      "Step: 24000,CV Accuracy = 99.18%\n",
      "Step: 25000,CV Accuracy = 99.22%\n",
      "Step: 26000,CV Accuracy = 99.26%\n",
      "Step: 27000,CV Accuracy = 99.26%\n",
      "Step: 28000,CV Accuracy = 99.26%\n",
      "Step: 29000,CV Accuracy = 99.26%\n",
      "Test Accuracy is 99.32%\n"
     ]
    }
   ],
   "source": [
    "val_data = {\n",
    "    x: mnist.validation.images,\n",
    "    y: mnist.validation.labels,\n",
    "    is_training: False\n",
    "}\n",
    "train_step = 30000\n",
    "for i in range(train_step):\n",
    "    xs, labels = mnist.train.next_batch(100)\n",
    "    LR= 0.8  # 0.001 =>0.1 => 0.5 =>0.8\n",
    "    sess.run(train_step, feed_dict={x: xs, y: labels,learning_rate:LR, is_training: True})\n",
    "    \n",
    "    if i % 1000 == 0:\n",
    "        acc = sess.run( accuracy, feed_dict= val_data)\n",
    "        print(\"Step: %5d,CV Accuracy = %5.2f%%\" % (i, acc * 100))\n",
    "\n",
    "test_data = {\n",
    "    x: mnist.test.images,\n",
    "    y: mnist.test.labels,\n",
    "    is_training: False\n",
    "}\n",
    "\n",
    "acc = sess.run(accuracy, feed_dict= test_data)\n",
    "\n",
    "print(\"Test Accuracy is %5.2f%%\" % (100 * acc))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "```\n",
    "  网络调参历程:  \n",
    "     CNN 基本结构复现至一篇Paper;(在此之前,根据基本网络添加卷积,池化,全连接层,实现的网络结构训练中欠拟合非常严重,多次调整结构后均不能有超过百分之50的Accuracy,随即找 Paper 看,从中取经)\n",
    "     网络结构基本 OK 后,在参数调整上,比较重要的参数首选是卷积核的 size和数量,这2个参数的调整对整个网络的影响都比较大;(size的设置调整中发现[3x3]或[5x5]的预测结果都非常接近,不过[3x3]的\n",
    "         收敛会更快;而采用[1x1]进行训练则会导致精确率下降到96%)\n",
    "     其次,优化器的学习率,训练中使用了两种方式来选择学习率,一种是是选择最大0.9 再逐渐衰减;二是由很小值0.001开始逐渐增大调整学习率;\n",
    "     在优化器选择上,也从 Moment Adam Adadelta  分别进行了实验,其中 Adadelta 效果最佳,收敛最快,Adam 反而一直没有找到有效的学习率,导致很难收敛;\n",
    "     dropout 正则化的设置对模型过拟合效果有一定的效果,由于我的训练过程中设置了训练验证集合,模型本身过拟合的问题也不大;\n",
    "     期间尝试过引入 BN层,在卷积层后加入 BN层,但模型的精确率反而变得比较低,这个原因也不是很明白!\n",
    "     略微比较奇怪的地方是,激活函数的引入:是否加入激活函数,对于训练结果的影响极其微弱,原因也是未知!\n",
    "  \n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "default_view": {},
   "name": "MNIST_Slim_CNN.ipynb",
   "provenance": [
    {
     "file_id": "https://github.com/stoensin/tensorflow-tutorial-1/blob/master/mnist-slim/MNIST_Slim_CNN.ipynb",
     "timestamp": 1532245652960
    }
   ],
   "version": "0.3.2",
   "views": {}
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
